#!/usr/bin/env python

import os, getopt, sys, time
import daemon
import threading
import Queue
import random
import signal

import dao
import logger

def sigint_handler(signum, frame):
    print 'got sigint, exit.'
    exit(1)

signal.signal(signal.SIGINT, sigint_handler)


queue = Queue.Queue()

def run_7k7k():
    table = '7k7k'
    print 'run for table %s' % (table,)
    

    for x in range(10):
        worker_7k7k(x).start()


    logger.debug('fill queue now')
    # fill queue
    max = dao.get_max_id(table)
    if not max:
        max = 1001
    print max

    random_ids = random.sample(range(1, max+1001), max+1000)

    for id in random_ids:
        queue.put(id)

    while not queue.empty():
        time.sleep(10)
    

class worker_7k7k(threading.Thread):

    def __init__(self, thread_id):
        threading.Thread.__init__(self)

        self.thread_id = thread_id
        self.table = '7k7k'

    def run(self):
        while True:
            try:
                id = queue.get()
            except Empty: 
                break
            
            logger.debug('worker 7k7k - %s: %s' % (self.thread_id, id))
            
            import sites.flash7k7k
            import dao
            try:
                if dao.need_refresh(self.table, id):
                    data = sites.flash7k7k.get_by_id(id)
                    dao.insert_row(self.table, data)
            except Exception, ex:
                logger.error('%s - %s' % (id, ex));

            queue.task_done()





def run_4399():
    table = '4399'
    print 'run for table %s' % (table,)
    

    for x in range(10):
        worker_4399(x).start()


    logger.debug('fill queue now')
    # fill queue
    max = dao.get_max_id(table)
    if not max:
        max = 1001
    print max

    random_ids = random.sample(range(1, max+1001), max+1000)

    for id in random_ids:
        queue.put(id)

    while not queue.empty():
        time.sleep(10)
    

class worker_4399(threading.Thread):

    def __init__(self, thread_id):
        threading.Thread.__init__(self)

        self.thread_id = thread_id
        self.table = '4399'

    def run(self):
        while True:
            try:
                id = queue.get()
            except Empty: 
                break
            
            logger.debug('worker 4399 - %s: %s' % (self.thread_id, id))
            
            import sites.flash4399
            import dao
            try:
                if dao.need_refresh(self.table, id):
                    data = sites.flash4399.get_by_id(id)
                    dao.insert_row(self.table, data)
            except Exception, ex:
                logger.error('%s - %s' % (id, ex));

            queue.task_done()



def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "t:rd")
    except getopt.GetoptError, err:
        print 'bad options'
        sys.exit(2)
    
    for o, a in opts:
        if o == "-d":
            daemon.daemonize('/tmp/crawler.pid')
        if o == "-r":
            # reload all
            pass

    xopts = dict(opts)
    table = xopts.get('-t')
    if table not in ('4399', '7k7k'):
        print 'bad requested table'
        exit(101)

    
    logger.debug('prepare run for %s' % table)

    eval('run_%s()' % table)



if __name__ == "__main__":
    main()



